# plotly standard imports
import plotly.graph_objs as go
import chart_studio.plotly as py

# Cufflinks wrapper on plotly
import cufflinks

# Data science imports
import pandas as pd
import numpy as np

# Options for pandas
pd.options.display.max_columns = 30

# Display all cell outputs
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = 'all'

from plotly.offline import iplot, init_notebook_mode
cufflinks.go_offline(connected=True)
init_notebook_mode(connected=True)

# Set global theme
cufflinks.set_config_file(world_readable=True, theme='pearl')

Prediction

Will try predict and denormalise model prediction

from src.load_datasets import load_datasets
from src.prepare_datasets import add_indicators

train, test = load_datasets()

test.index = pd.to_datetime(test.pop('timestamp'), unit='ms')

test
test[::60].iplot(subplots=True)

	open	high	low	close	volume
timestamp
2020-04-22 12:42:00	6954.70000	6956.5	6954.600000	6956.5	0.336500
2020-04-22 12:43:00	6956.20000	6957.6	6952.800000	6955.5	0.038800
2020-04-22 12:44:00	6956.30000	6956.3	6955.779638	6956.0	0.300000
2020-04-22 12:45:00	6956.10000	6957.5	6955.300000	6955.3	0.113594
2020-04-22 12:46:00	6955.80000	6955.8	6955.300000	6955.3	0.016867
...	...	...	...	...	...
2021-03-09 18:50:00	53725.00000	53727.0	53725.000000	53727.0	1.119537
2021-03-09 18:51:00	53780.00000	53780.0	53779.000000	53779.0	0.010000
2021-03-09 18:52:00	53814.00000	53828.0	53814.000000	53828.0	0.016450
2021-03-09 18:53:00	53903.00000	53910.0	53902.000000	53910.0	0.012500
2021-03-09 18:54:00	53930.36601	53958.0	53930.366010	53953.0	1.040000

454074 rows × 5 columns

test = add_indicators(test)
test = test.dropna()

test.head()
test.iplot(subplots=True)

/usr/local/lib/python3.6/dist-packages/ta/trend.py:768: RuntimeWarning:

invalid value encountered in double_scalars

/usr/local/lib/python3.6/dist-packages/ta/trend.py:772: RuntimeWarning:

invalid value encountered in double_scalars

	open	high	low	close	volume	volatility_bbm	volatility_bbh	volatility_bbl	trend_macd	momentum_rsi	volatility_kchi	trend_ichimoku_conv	trend_ichimoku_a	trend_ichimoku_b	momentum_stoch	momentum_stoch_signal	volatility_atr
timestamp
2020-04-22 12:42:00	6954.7	6956.5	6954.600000	6956.5	0.336500	6956.500	6956.500000	6956.500000	0.000000	100.000000	0.0	6955.55	6955.55	6955.55	100.000000	100.000000	0.0
2020-04-22 12:43:00	6956.2	6957.6	6952.800000	6955.5	0.038800	6956.000	6957.000000	6955.000000	-0.079772	0.000000	0.0	6955.20	6955.20	6955.20	56.250000	78.125000	0.0
2020-04-22 12:44:00	6956.3	6956.3	6955.779638	6956.0	0.300000	6956.000	6956.816497	6955.183503	-0.101476	35.000000	0.0	6955.20	6955.20	6955.20	66.666667	74.305556	0.0
2020-04-22 12:45:00	6956.1	6957.5	6955.300000	6955.3	0.113594	6955.825	6956.756397	6954.893603	-0.173165	22.910373	0.0	6955.20	6955.20	6955.20	52.083333	58.333333	0.0
2020-04-22 12:46:00	6955.8	6955.8	6955.300000	6955.3	0.016867	6955.720	6956.652952	6954.787048	-0.227358	22.910373	0.0	6955.20	6955.20	6955.20	52.083333	56.944444	0.0

from tqdm import tqdm
from src.prepare_datasets import get_prepared_datasets

train_norm, test_norm = get_prepared_datasets()

train_norm[::60].iplot(subplots=True)

test_norm = test_norm[test.columns.tolist()]

test_norm[::60].iplot(subplots=True)

Check denormalisation working correctly

from src.prepare_datasets import get_scaler

scaler = get_scaler()

test_denorm = pd.DataFrame(scaler.inverse_transform(test_norm))
test_denorm.columns = test_norm.columns
test_denorm.index = test.index

test_denorm[::60].iplot(subplots=True)

test.head()
test_norm.head()
test_denorm.head()

	open	high	low	close	volume	volatility_bbm	volatility_bbh	volatility_bbl	trend_macd	momentum_rsi	volatility_kchi	trend_ichimoku_conv	trend_ichimoku_a	trend_ichimoku_b	momentum_stoch	momentum_stoch_signal	volatility_atr
timestamp
2020-04-22 12:42:00	6954.7	6956.5	6954.600000	6956.5	0.336500	6956.500	6956.500000	6956.500000	0.000000	100.000000	0.0	6955.55	6955.55	6955.55	100.000000	100.000000	0.0
2020-04-22 12:43:00	6956.2	6957.6	6952.800000	6955.5	0.038800	6956.000	6957.000000	6955.000000	-0.079772	0.000000	0.0	6955.20	6955.20	6955.20	56.250000	78.125000	0.0
2020-04-22 12:44:00	6956.3	6956.3	6955.779638	6956.0	0.300000	6956.000	6956.816497	6955.183503	-0.101476	35.000000	0.0	6955.20	6955.20	6955.20	66.666667	74.305556	0.0
2020-04-22 12:45:00	6956.1	6957.5	6955.300000	6955.3	0.113594	6955.825	6956.756397	6954.893603	-0.173165	22.910373	0.0	6955.20	6955.20	6955.20	52.083333	58.333333	0.0
2020-04-22 12:46:00	6955.8	6955.8	6955.300000	6955.3	0.016867	6955.720	6956.652952	6954.787048	-0.227358	22.910373	0.0	6955.20	6955.20	6955.20	52.083333	56.944444	0.0

	open	high	low	close	volume	volatility_bbm	volatility_bbh	volatility_bbl	trend_macd	momentum_rsi	volatility_kchi	trend_ichimoku_conv	trend_ichimoku_a	trend_ichimoku_b	momentum_stoch	momentum_stoch_signal	volatility_atr
0	0.064278	0.064295	0.064278	0.064297	0.000050	0.331075	0.330149	0.331435	0.588327	1.000000	0.0	0.330995	0.331188	0.332033	1.000000	1.000000	0.0
1	0.064293	0.064306	0.064260	0.064287	0.000006	0.331050	0.330175	0.331357	0.588211	-0.001679	0.0	0.330977	0.331170	0.332015	0.562500	0.781250	0.0
2	0.064294	0.064293	0.064290	0.064292	0.000045	0.331050	0.330166	0.331367	0.588179	0.348909	0.0	0.330977	0.331170	0.332015	0.666667	0.743056	0.0
3	0.064292	0.064305	0.064285	0.064285	0.000017	0.331041	0.330162	0.331352	0.588074	0.227810	0.0	0.330977	0.331170	0.332015	0.520833	0.583333	0.0
4	0.064289	0.064288	0.064285	0.064285	0.000003	0.331035	0.330157	0.331346	0.587995	0.227810	0.0	0.330977	0.331170	0.332015	0.520833	0.569444	0.0

	open	high	low	close	volume	volatility_bbm	volatility_bbh	volatility_bbl	trend_macd	momentum_rsi	volatility_kchi	trend_ichimoku_conv	trend_ichimoku_a	trend_ichimoku_b	momentum_stoch	momentum_stoch_signal	volatility_atr
timestamp
2020-04-22 12:42:00	6954.7	6956.5	6954.600000	6956.5	0.336500	6956.500	6956.500000	6956.500000	0.000000	100.000000	0.0	6955.55	6955.55	6955.55	100.000000	100.000000	0.0
2020-04-22 12:43:00	6956.2	6957.6	6952.800000	6955.5	0.038800	6956.000	6957.000000	6955.000000	-0.079772	0.000000	0.0	6955.20	6955.20	6955.20	56.250000	78.125000	0.0
2020-04-22 12:44:00	6956.3	6956.3	6955.779638	6956.0	0.300000	6956.000	6956.816497	6955.183503	-0.101476	35.000000	0.0	6955.20	6955.20	6955.20	66.666667	74.305556	0.0
2020-04-22 12:45:00	6956.1	6957.5	6955.300000	6955.3	0.113594	6955.825	6956.756397	6954.893603	-0.173165	22.910373	0.0	6955.20	6955.20	6955.20	52.083333	58.333333	0.0
2020-04-22 12:46:00	6955.8	6955.8	6955.300000	6955.3	0.016867	6955.720	6956.652952	6954.787048	-0.227358	22.910373	0.0	6955.20	6955.20	6955.20	52.083333	56.944444	0.0

from src.libs import params, prepare, save_metrics, load, checkpoints
from src.model import build_model
from src.prepare_datasets import make_generator

model = build_model()
model = checkpoints.load_weights(model)

len(test_norm)

for_pred = test_norm[:5000]

ds = make_generator(test_norm, test_norm[['close']], shuffle=False)

predictions = model.predict(ds)

len(predictions)
predictions.shape

454074

---------------------------------------------------------------------------
InternalError                             Traceback (most recent call last)
<ipython-input-5-639f1c23f52f> in <module>
     12 ds = make_generator(test_norm, test_norm[['close']], shuffle=False)
     13 
---> 14 predictions = model.predict(ds)
     15 
     16 len(predictions)

/usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/engine/training.py in predict(self, x, batch_size, verbose, steps, callbacks, max_queue_size, workers, use_multiprocessing)
   1627           for step in data_handler.steps():
   1628             callbacks.on_predict_batch_begin(step)
-> 1629             tmp_batch_outputs = self.predict_function(iterator)
   1630             if data_handler.should_sync:
   1631               context.async_wait()

/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py in __call__(self, *args, **kwds)
    826     tracing_count = self.experimental_get_tracing_count()
    827     with trace.Trace(self._name) as tm:
--> 828       result = self._call(*args, **kwds)
    829       compiler = "xla" if self._experimental_compile else "nonXla"
    830       new_tracing_count = self.experimental_get_tracing_count()

/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)
    886         # Lifting succeeded, so variables are initialized and we can run the
    887         # stateless function.
--> 888         return self._stateless_fn(*args, **kwds)
    889     else:
    890       _, _, _, filtered_flat_args = \

/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py in __call__(self, *args, **kwargs)
   2941        filtered_flat_args) = self._maybe_define_function(args, kwargs)
   2942     return graph_function._call_flat(
-> 2943         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access
   2944 
   2945   @property

/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py in _call_flat(self, args, captured_inputs, cancellation_manager)
   1917       # No tape is watching; skip to running the function.
   1918       return self._build_call_outputs(self._inference_function.call(
-> 1919           ctx, args, cancellation_manager=cancellation_manager))
   1920     forward_backward = self._select_forward_and_backward_functions(
   1921         args,

/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/function.py in call(self, ctx, args, cancellation_manager)
    558               inputs=args,
    559               attrs=attrs,
--> 560               ctx=ctx)
    561         else:
    562           outputs = execute.execute_with_cancellation(

/usr/local/lib/python3.6/dist-packages/tensorflow/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
     58     ctx.ensure_initialized()
     59     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 60                                         inputs, attrs, num_outputs)
     61   except core._NotOkStatusException as e:
     62     if name is not None:

InternalError:    Failed to call ThenRnnForward with model config: [rnn_mode, rnn_input_mode, rnn_direction_mode]: 2, 0, 0 , [num_layers, input_size, num_units, dir_count, max_seq_length, batch_size, cell_num_units]: [1, 17, 128, 1, 150, 256, 128] 
	 [[{{node CudnnRNN}}]]
	 [[sequential/lstm/PartitionedCall]] [Op:__inference_predict_function_1726]

Function call stack:
predict_function -> predict_function -> predict_function

input_w, label_w = next(iter(ds))

len(input_w)
input_w.shape

# load if memory not enough
predictions = pd.read_csv('./predictions.csv')
predictions.index = test[-len(predictions):].index

predictions = predictions[['0']]
predictions

	0
timestamp
2020-04-22 17:42:00	-0.005181
2020-04-22 17:43:00	-0.005930
2020-04-22 17:44:00	-0.006075
2020-04-22 17:45:00	0.006286
2020-04-22 17:46:00	0.009975
...	...
2021-03-09 18:50:00	0.048660
2021-03-09 18:51:00	0.050219
2021-03-09 18:52:00	0.050700
2021-03-09 18:53:00	0.049691
2021-03-09 18:54:00	0.041920

453775 rows × 1 columns

predictions[::60].iplot()

len(predictions)

predictions.head()

453775

	0
timestamp
2020-04-22 17:42:00	-0.005181
2020-04-22 17:43:00	-0.005930
2020-04-22 17:44:00	-0.006075
2020-04-22 17:45:00	0.006286
2020-04-22 17:46:00	0.009975

predictions['0']

timestamp
2020-04-22 17:42:00   -0.005181
2020-04-22 17:43:00   -0.005930
2020-04-22 17:44:00   -0.006075
2020-04-22 17:45:00    0.006286
2020-04-22 17:46:00    0.009975
                         ...   
2021-03-09 18:50:00    0.048660
2021-03-09 18:51:00    0.050219
2021-03-09 18:52:00    0.050700
2021-03-09 18:53:00    0.049691
2021-03-09 18:54:00    0.041920
Name: 0, Length: 453775, dtype: float64

for_inversion = test_norm[-len(predictions):]
for_inversion.index = predictions.index
for_inversion['close'] = predictions['0']

for_inversion.head()

for_inversion[::60].iplot(subplots=True)

	open	high	low	close	volume	volatility_bbm	volatility_bbh	volatility_bbl	trend_macd	momentum_rsi	volatility_kchi	trend_ichimoku_conv	trend_ichimoku_a	trend_ichimoku_b	momentum_stoch	momentum_stoch_signal	volatility_atr
timestamp
2020-04-22 17:42:00	0.065985	0.065985	0.065986	-0.005181	0.000163	0.340031	0.339520	0.339957	0.585971	0.365318	0.0	0.340032	0.340228	0.341360	0.101351	0.124196	0.006263
2020-04-22 17:43:00	0.065986	0.065985	0.065987	-0.005930	0.000014	0.340014	0.339515	0.339928	0.585727	0.365318	0.0	0.339914	0.340180	0.341357	0.101351	0.110360	0.005670
2020-04-22 17:44:00	0.065986	0.065984	0.065965	-0.006075	0.000624	0.340008	0.339524	0.339907	0.585319	0.327473	0.0	0.339891	0.340160	0.341303	0.000000	0.067568	0.005806
2020-04-22 17:45:00	0.065951	0.065950	0.065908	0.006286	0.000319	0.339988	0.339572	0.339818	0.584372	0.251997	0.0	0.339716	0.339999	0.341227	0.000000	0.033784	0.007108
2020-04-22 17:46:00	0.065900	0.065899	0.065694	0.009975	0.003379	0.339939	0.339729	0.339562	0.582381	0.169425	0.0	0.339083	0.339407	0.340734	0.240566	0.080189	0.013458

denorm_pred = pd.DataFrame(scaler.inverse_transform(for_inversion))

denorm_pred.columns = for_inversion.columns
denorm_pred.index = for_inversion.index

len(denorm_pred)

denorm_pred.head()

453775

	open	high	low	close	volume	volatility_bbm	volatility_bbh	volatility_bbl	trend_macd	momentum_rsi	volatility_kchi	trend_ichimoku_conv	trend_ichimoku_a	trend_ichimoku_b	momentum_stoch	momentum_stoch_signal	volatility_atr
timestamp
2020-04-22 17:42:00	7124.400000	7124.600000	7124.400000	47.791795	1.094616	7129.420469	7137.942772	7120.898165	-1.613397	36.638202	0.0	7130.079106	7130.039553	7135.10	10.135135	12.419562	1.889362
2020-04-22 17:43:00	7124.500000	7124.600000	7124.500000	-26.640479	0.091899	7129.100469	7137.854008	7120.346929	-1.779975	36.638202	0.0	7127.800000	7129.100000	7135.05	10.135135	11.036036	1.710426
2020-04-22 17:44:00	7124.500000	7124.500000	7122.379928	-41.122185	4.194133	7128.979465	7138.029339	7119.929591	-2.059323	32.859994	0.0	7127.355716	7128.722840	7134.00	0.000000	6.756757	1.751390
2020-04-22 17:45:00	7121.100000	7121.100000	7116.700000	1188.078582	2.142919	7128.589465	7138.954586	7118.224344	-2.707817	25.325065	0.0	7123.975579	7125.612790	7132.55	0.000000	3.378378	2.144244
2020-04-22 17:46:00	7116.014573	7116.014573	7095.400000	1554.856630	22.697481	7127.644465	7142.004897	7113.284033	-4.070508	17.081660	0.0	7111.750000	7114.175000	7123.05	24.056604	8.018868	4.059820

predicted2norm = pd.DataFrame({
    'predicted': denorm_pred['close'],
    'real': test['close'][-len(denorm_pred):]
})

predicted2norm.head()

predicted2norm.index = test[-len(denorm_pred):].index

predicted2norm[::60].iplot()

	predicted	real
timestamp
2020-04-22 17:42:00	47.791795	7124.500000
2020-04-22 17:43:00	-26.640479	7124.500000
2020-04-22 17:44:00	-41.122185	7122.379928
2020-04-22 17:45:00	1188.078582	7116.700000
2020-04-22 17:46:00	1554.856630	7105.600000